package twitter;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;

import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;


public class Tokenizador {
	
	InputStream modelIn;
	TokenizerModel model;
	Tokenizer tokenizer;
	
	public Tokenizador(){
		try {
			modelIn = new FileInputStream("en-token.bin");
		} catch (FileNotFoundException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
	
	    try {
	      model = new TokenizerModel(modelIn);
	    }
	    catch (IOException e) {
	      e.printStackTrace();
	    }
	    finally {
	      if (modelIn != null) {
	        try {
	          modelIn.close();
	        }
	        catch (IOException e) {
	        }
	      }
	    }
	    tokenizer = new TokenizerME(model);
	}
	
	public String[] tokenizar(String frase){
		String[] tokens = null;
		tokens = tokenizer.tokenize(frase);
		return tokens;
		
	}
	
}
